﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Resources;
using System.Text;

using Microsoft.Build.Framework;
using Microsoft.Build.Shared;

namespace Microsoft.Build.Utilities
{
    public class FlatTrackingData
    {
        #region Constants
        // The maximum number of outputs that should be logged, if more than this, then no outputs are logged
        private const int MaxLogCount = 100;
        #endregion

        #region Member Data
        // The output dependency table
        private IDictionary<string, DateTime> _dependencyTable = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
        // The .write. trackg log files
        private ITaskItem[] _tlogFiles;

        // The tlog marker is used if the tracking data is empty
        // even if the tracked execution was successful
        private string _tlogMarker = String.Empty;

        // The TaskLoggingHelper that we log progress to
        private TaskLoggingHelper _log;
        // Are the tracking logs that we were constructed with actually available
        private bool _tlogsAvailable;

        // The oldest file that we have seen
        private string _oldestFileName = String.Empty;
        private DateTime _oldestFileTimeUtc = DateTime.MaxValue;

        // The newest file what we have seen
        private string _newestFileName = String.Empty;
        private DateTime _newestFileTimeUtc = DateTime.MinValue;

        // Should rooting markers be treated as tracking entries
        private bool _treatRootMarkersAsEntries = false;

        // If files are missing when reading the Tlog, skip them
        private bool _skipMissingFiles = false;
        // If we are not skipping missing files, what DateTime should they be given?
        private DateTime _missingFileTimeUtc = DateTime.MinValue;
        // Missing files that have been detected in the TLog
        private List<string> _missingFiles = new List<string>();

        // The newest Tlog that we have seen
        private DateTime _newestTLogTimeUtc = DateTime.MinValue;
        private string _newestTLogFileName = String.Empty;

        // Cache of last write times
        private IDictionary<string, DateTime> _lastWriteTimeUtcCache = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);

        // The set of paths that contain files that are to be ignored during up to date check - these directories or their subdirectories
        private List<string> _excludedInputPaths = new List<string>();
        #endregion

        #region Properties

        // Provide external access to the dependencyTable
#if WHIDBEY_VISIBILITY
        internal
#else
        public
#endif
        IDictionary<string, DateTime> DependencyTable
        {
            get { return _dependencyTable; }
        }

        /// <summary>
        /// Missing files have been detected in the TLog
        /// </summary>
        [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Justification = "Has shipped as public API, so we can't easily change it now. ")]
        public List<string> MissingFiles
        {
            get { return _missingFiles; }
            set { _missingFiles = value; }
        }

        /// <summary>
        /// The path for the oldest file we have seen
        /// </summary>
        public string OldestFileName
        {
            get { return _oldestFileName; }
            set { _oldestFileName = value; }
        }

        /// <summary>
        /// The time for the oldest file we have seen
        /// </summary>
        public DateTime OldestFileTime
        {
            get { return _oldestFileTimeUtc.ToLocalTime(); }
            set { _oldestFileTimeUtc = value.ToUniversalTime(); }
        }

        /// <summary>
        /// The time for the oldest file we have seen
        /// </summary>
        public DateTime OldestFileTimeUtc
        {
            get { return _oldestFileTimeUtc; }
            set { _oldestFileTimeUtc = value.ToUniversalTime(); }
        }

        /// <summary>
        /// The path for the newest file we have seen
        /// </summary>
        public string NewestFileName
        {
            get { return _newestFileName; }
            set { _newestFileName = value; }
        }

        /// <summary>
        /// The time for the newest file we have seen
        /// </summary>
        public DateTime NewestFileTime
        {
            get { return _newestFileTimeUtc.ToLocalTime(); }
            set { _newestFileTimeUtc = value.ToUniversalTime(); }
        }

        /// <summary>
        /// The time for the newest file we have seen
        /// </summary>
        public DateTime NewestFileTimeUtc
        {
            get { return _newestFileTimeUtc; }
            set { _newestFileTimeUtc = value.ToUniversalTime(); }
        }

        /// <summary>
        /// Should root markers in the TLog be treated as file accesses, or only as markers?
        /// </summary>
        public bool TreatRootMarkersAsEntries
        {
            get { return _treatRootMarkersAsEntries; }
            set { _treatRootMarkersAsEntries = value; }
        }

        /// <summary>
        /// Should files in the TLog but no longer exist be skipped or recorded?
        /// </summary>
        public bool SkipMissingFiles
        {
            get { return _skipMissingFiles; }
            set { _skipMissingFiles = value; }
        }

        /// <summary>
        /// The TLog files that back this structure
        /// </summary>
        [SuppressMessage("Microsoft.Performance", "CA1819:PropertiesShouldNotReturnArrays", Justification = "Has shipped as public API, so we can't easily change it now. ")]
        public ITaskItem[] TlogFiles
        {
            get { return _tlogFiles; }
            set { _tlogFiles = value; }
        }

        /// <summary>
        /// The time of the newest Tlog
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public DateTime NewestTLogTime
        {
            get { return _newestTLogTimeUtc.ToLocalTime(); }
            set { _newestTLogTimeUtc = value.ToUniversalTime(); }
        }

        /// <summary>
        /// The time of the newest Tlog
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public DateTime NewestTLogTimeUtc
        {
            get { return _newestTLogTimeUtc; }
            set { _newestTLogTimeUtc = value.ToUniversalTime(); }
        }

        /// <summary>
        /// The path of the newest TLog file
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public string NewestTLogFileName
        {
            get { return _newestTLogFileName; }
            set { _newestTLogFileName = value; }
        }

        /// <summary>
        /// Are all the TLogs that were passed to us actually available on disk?
        /// </summary>
        public bool TlogsAvailable
        {
            get { return _tlogsAvailable; }
            set { _tlogsAvailable = value; }
        }

        #endregion

        #region Constructors
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, DateTime missingFileTimeUtc)
        {
            InternalConstruct(null, tlogFiles, null, false, missingFileTimeUtc, null);
        }

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, ITaskItem[] tlogFilesToIgnore, DateTime missingFileTimeUtc)
        {
            InternalConstruct(null, tlogFiles, tlogFilesToIgnore, false, missingFileTimeUtc, null);
        }

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .tlog files to interpret</param>
        /// <param name="tlogFilesToIgnore">The .tlog files to ignore</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        /// <param name="excludedInputPaths">The set of paths that contain files that are to be ignored during up to date check, including any subdirectories.</param>
        /// <param name="sharedLastWriteTimeUtcCache">Cache to be used for all timestamp/exists comparisons, which can be shared between multiple FlatTrackingData instances.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, ITaskItem[] tlogFilesToIgnore, DateTime missingFileTimeUtc, string[] excludedInputPaths, IDictionary<string, DateTime> sharedLastWriteTimeUtcCache)
        {
            if (sharedLastWriteTimeUtcCache != null)
            {
                _lastWriteTimeUtcCache = sharedLastWriteTimeUtcCache;
            }

            InternalConstruct(null, tlogFiles, tlogFilesToIgnore, false, missingFileTimeUtc, excludedInputPaths);
        }


        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFiles">The tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        public FlatTrackingData(ITask ownerTask, ITaskItem[] tlogFiles, DateTime missingFileTimeUtc)
        {
            InternalConstruct(ownerTask, tlogFiles, null, false, missingFileTimeUtc, null);
        }

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, bool skipMissingFiles)
        {
            InternalConstruct(null, tlogFiles, null, skipMissingFiles, DateTime.MinValue, null);
        }

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFiles">The tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        public FlatTrackingData(ITask ownerTask, ITaskItem[] tlogFiles, bool skipMissingFiles)
        {
            InternalConstruct(ownerTask, tlogFiles, null, skipMissingFiles, DateTime.MinValue, null);
        }

        /// <summary>
        /// Internal constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        /// <param name="excludedInputPaths">The set of paths that contain files that are to be ignored during up to date check</param>
        private void InternalConstruct(ITask ownerTask, ITaskItem[] tlogFilesLocal, ITaskItem[] tlogFilesToIgnore, bool skipMissingFiles, DateTime missingFileTimeUtc, string[] excludedInputPaths)
        {
            if (ownerTask != null)
            {
                _log = new TaskLoggingHelper(ownerTask);
                _log.TaskResources = AssemblyResources.PrimaryResources;
                _log.HelpKeywordPrefix = "MSBuild.";
            }

            ITaskItem[] expandedTlogFiles = TrackedDependencies.ExpandWildcards(tlogFilesLocal);

            if (tlogFilesToIgnore != null)
            {
                ITaskItem[] expandedTlogFilesToIgnore = TrackedDependencies.ExpandWildcards(tlogFilesToIgnore);

                if (expandedTlogFilesToIgnore.Length > 0)
                {
                    HashSet<string> ignore = new HashSet<string>();
                    List<ITaskItem> remainingTlogFiles = new List<ITaskItem>();

                    foreach (ITaskItem tlogFileToIgnore in expandedTlogFilesToIgnore)
                    {
                        ignore.Add(tlogFileToIgnore.ItemSpec);
                    }

                    foreach (ITaskItem tlogFile in expandedTlogFiles)
                    {
                        if (!ignore.Contains(tlogFile.ItemSpec))
                        {
                            remainingTlogFiles.Add(tlogFile);
                        }
                    }

                    _tlogFiles = remainingTlogFiles.ToArray();
                }
                else
                {
                    _tlogFiles = expandedTlogFiles;
                }
            }
            else
            {
                _tlogFiles = expandedTlogFiles;
            }

            // We have no TLog files on disk, create a TLog marker from the
            // TLogFiles ItemSpec so we can fabricate one if we need to
            // This becomes our "first" tlog, since on the very first run, no tlogs
            // will exist, and if a compaction has been run (as part of the initial up-to-date check) then this
            // marker tlog will be created as empty.
            if (_tlogFiles == null || _tlogFiles.Length == 0)
            {
                _tlogMarker = tlogFilesLocal[0].ItemSpec.Replace("*", "1");
                _tlogMarker = _tlogMarker.Replace("?", "2");
            }

            if (excludedInputPaths != null)
            {
                // Assign our exclude paths to our lookup - and make sure that all recorded paths end in a slash so that
                // our "starts with" comparison doesn't pick up incomplete matches, such as C:\Foo matching C:\FooFile.txt
                foreach (string excludePath in excludedInputPaths)
                {
                    string fullexcludePath = FileUtilities.EnsureTrailingSlash(FileUtilities.NormalizePath(excludePath)).ToUpperInvariant();
                    _excludedInputPaths.Add(fullexcludePath);
                }
            }

            _tlogsAvailable = TrackedDependencies.ItemsExist(_tlogFiles);
            _skipMissingFiles = skipMissingFiles;
            _missingFileTimeUtc = missingFileTimeUtc.ToUniversalTime();
            if (_tlogFiles != null)
            {
                // Read the TLogs into our internal structures
                ConstructFileTable();
            }
        }
        #endregion

        #region Methods
        /// <summary>
        /// Construct our dependency table for our source files
        /// </summary>
        private void ConstructFileTable()
        {
            string tLogRootingMarker = null;
            try
            {
                // construct a rooting marker from the tlog files
                tLogRootingMarker = DependencyTableCache.FormatNormalizedTlogRootingMarker(_tlogFiles);
            }
            catch (ArgumentException e)
            {
                FileTracker.LogWarningWithCodeFromResources(_log, "Tracking_RebuildingDueToInvalidTLog", e.Message);
                return;
            }
            if (!_tlogsAvailable)
            {
                lock (DependencyTableCache.DependencyTable)
                {
                    // The tracking logs are not available, they may have been deleted at some point.
                    // Be safe and remove any references from the cache.
                    if (DependencyTableCache.DependencyTable.ContainsKey(tLogRootingMarker))
                    {
                        DependencyTableCache.DependencyTable.Remove(tLogRootingMarker);
                    }
                }
                return;
            }

            DependencyTableCacheEntry cachedEntry = null;

            lock (DependencyTableCache.DependencyTable)
            {
                // Look in the dependency table cache to see if its available and up to date
                cachedEntry = DependencyTableCache.GetCachedEntry(tLogRootingMarker);
            }

            // We have an up to date cached entry
            if (cachedEntry != null)
            {
                _dependencyTable = (IDictionary<string, DateTime>)cachedEntry.DependencyTable;

                // We may have stored the dependency table in the cache, but all the other information
                // (newest file time, number of missing files, etc.) has been reset to default.  Refresh
                // the data.  
                this.UpdateFileEntryDetails();

                // Log information about what we're using
                FileTracker.LogMessageFromResources(_log, MessageImportance.Low, "Tracking_TrackingCached");
                foreach (ITaskItem tlogItem in cachedEntry.TlogFiles)
                {
                    FileTracker.LogMessage(_log, MessageImportance.Low, "\t{0}", tlogItem.ItemSpec);
                }
                return;
            }

            FileTracker.LogMessageFromResources(_log, MessageImportance.Low, "Tracking_TrackingLogs");
            // Now we need to construct the rest of the table from the TLOG files
            // If there are any errors in the tlogs, we want to warn, stop parsing tlogs, and empty 
            // out the dependency table, essentially forcing a rebuild.  
            bool encounteredInvalidTLogContents = false;
            bool exceptionCaught = false;
            string invalidTLogName = null;
            foreach (ITaskItem tlogFileName in _tlogFiles)
            {
                try
                {
                    FileTracker.LogMessage(_log, MessageImportance.Low, "\t{0}", tlogFileName.ItemSpec);

                    DateTime tlogLastWriteTimeUtc = NativeMethods.GetLastWriteTimeUtc(tlogFileName.ItemSpec);
                    if (tlogLastWriteTimeUtc > _newestTLogTimeUtc)
                    {
                        _newestTLogTimeUtc = tlogLastWriteTimeUtc;
                        _newestTLogFileName = tlogFileName.ItemSpec;
                    }

                    using (StreamReader tlog = File.OpenText(tlogFileName.ItemSpec))
                    {
                        string tlogEntry = tlog.ReadLine();

                        while (tlogEntry != null)
                        {
                            if (tlogEntry.Length == 0) // empty lines are a sign that something has gone wrong
                            {
                                encounteredInvalidTLogContents = true;
                                invalidTLogName = tlogFileName.ItemSpec;
                                break;
                            }
                            // Preprocessing for the line entry
                            else if (tlogEntry[0] == '#') // a comment marker should be skipped
                            {
                                tlogEntry = tlog.ReadLine();
                                continue;
                            }
                            else if (tlogEntry[0] == '^' && TreatRootMarkersAsEntries) // This is a rooting record, and we should keep it
                            {
                                tlogEntry = tlogEntry.Substring(1);

                                if (tlogEntry.Length == 0)
                                {
                                    encounteredInvalidTLogContents = true;
                                    invalidTLogName = tlogFileName.ItemSpec;
                                    break;
                                }
                            }
                            else if (tlogEntry[0] == '^') // root marker is not being treated as an entry, skip it
                            {
                                tlogEntry = tlog.ReadLine();
                                continue;
                            }

                            // If we haven't seen this file before, then record it
                            if (!_dependencyTable.ContainsKey(tlogEntry))
                            {
                                // It may be that this is one of the locations that we should ignore
                                if (!FileTracker.FileIsExcludedFromDependencies(tlogEntry))
                                {
                                    RecordEntryDetails(tlogEntry, true);
                                }
                            }
                            tlogEntry = tlog.ReadLine();
                        }
                    }
                }
                catch (Exception e)
                {
                    if (ExceptionHandling.NotExpectedException(e))
                    {
                        throw;
                    }

                    FileTracker.LogWarningWithCodeFromResources(_log, "Tracking_RebuildingDueToInvalidTLog", e.Message);
                    break;
                }

                if (encounteredInvalidTLogContents)
                {
                    FileTracker.LogWarningWithCodeFromResources(_log, "Tracking_RebuildingDueToInvalidTLogContents", invalidTLogName);
                    break;
                }
            }

            lock (DependencyTableCache.DependencyTable)
            {
                // There were problems with the tracking logs -- we've already warned or errored; now we want to make 
                // sure that we essentially force a rebuild of this particular root. 
                if (encounteredInvalidTLogContents || exceptionCaught)
                {
                    if (DependencyTableCache.DependencyTable.ContainsKey(tLogRootingMarker))
                    {
                        DependencyTableCache.DependencyTable.Remove(tLogRootingMarker);
                    }

                    _dependencyTable = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
                }
                else
                {
                    // Record the newly built dependency table in the cache
                    DependencyTableCache.DependencyTable[tLogRootingMarker] = new DependencyTableCacheEntry(_tlogFiles, (IDictionary)_dependencyTable);
                }
            }
        }

        /// <summary>
        /// Update the current state of entry details for the dependency table
        /// </summary>
        public void UpdateFileEntryDetails()
        {
            _oldestFileName = String.Empty;
            _oldestFileTimeUtc = DateTime.MaxValue;

            _newestFileName = String.Empty;
            _newestFileTimeUtc = DateTime.MinValue;

            _newestTLogFileName = String.Empty;
            _newestTLogTimeUtc = DateTime.MinValue;

            this.MissingFiles.Clear();

            // First update the details of our Tlogs
            foreach (ITaskItem tlogFileName in _tlogFiles)
            {
                DateTime tlogLastWriteTimeUtc = NativeMethods.GetLastWriteTimeUtc(tlogFileName.ItemSpec);
                if (tlogLastWriteTimeUtc > _newestTLogTimeUtc)
                {
                    _newestTLogTimeUtc = tlogLastWriteTimeUtc;
                    _newestTLogFileName = tlogFileName.ItemSpec;
                }
            }

            // Now for each entry in the table
            foreach (string entry in this.DependencyTable.Keys)
            {
                RecordEntryDetails(entry, false);
            }
        }

        /// <summary>
        /// Test to see if the specified file is excluded from tracked dependency checking
        /// </summary>
        /// <param name="fileName">
        /// Full path of the file to test
        /// </param>
        /// <remarks>
        /// The file is excluded if it is within any of the specified excluded input paths or any subdirectory of the paths.
        /// It also assumes the file name is already converted to Uppercase Invariant.
        /// </remarks>
        public bool FileIsExcludedFromDependencyCheck(string fileName)
        {
            foreach (string path in _excludedInputPaths)
            {
                if (fileName.StartsWith(path, StringComparison.OrdinalIgnoreCase))
                {
                    return true;
                }
            }

            return false;
        }

        /// <summary>
        /// Record the time and missing state of the entry in the tlog
        /// </summary>
        /// <param name="tlogEntry"></param>
        private void RecordEntryDetails(string tlogEntry, bool populateTable)
        {
            if (FileIsExcludedFromDependencyCheck(tlogEntry))
            {
                return;
            }

            DateTime fileModifiedTimeUtc = GetLastWriteTimeUtc(tlogEntry);
            if (_skipMissingFiles && fileModifiedTimeUtc == DateTime.MinValue) // the file is missing
            {
                return;
            }
            else if (fileModifiedTimeUtc == DateTime.MinValue)
            {
                // Record the file in our table even though it was missing
                // use the missingFileTimeUtc as indicated.
                if (populateTable)
                {
                    _dependencyTable[tlogEntry] = _missingFileTimeUtc.ToUniversalTime();
                }
                _missingFiles.Add(tlogEntry);
            }
            else
            {
                if (populateTable)
                {
                    _dependencyTable[tlogEntry] = fileModifiedTimeUtc;
                }
            }

            // Record this file if it is newer than our current newest
            if (fileModifiedTimeUtc > _newestFileTimeUtc)
            {
                _newestFileTimeUtc = fileModifiedTimeUtc;
                _newestFileName = tlogEntry;
            }

            // Record this file if it is older than our current oldest
            if (fileModifiedTimeUtc < _oldestFileTimeUtc)
            {
                _oldestFileTimeUtc = fileModifiedTimeUtc;
                _oldestFileName = tlogEntry;
            }
        }

        /// <summary>
        /// This method will re-write the tlogs from the output table
        /// </summary>
        public void SaveTlog()
        {
            SaveTlog(null);
        }

        /// <summary>
        /// This method will re-write the tlogs from the current table
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public void SaveTlog(DependencyFilter includeInTLog)
        {
            if (_tlogFiles != null && (_tlogFiles.Length > 0))
            {
                string tLogRootingMarker = DependencyTableCache.FormatNormalizedTlogRootingMarker(_tlogFiles);

                lock (DependencyTableCache.DependencyTable)
                {
                    // The tracking logs in the cache will be invalidated by this write
                    // remove the cached entries to be sure
                    if (DependencyTableCache.DependencyTable.ContainsKey(tLogRootingMarker))
                    {
                        DependencyTableCache.DependencyTable.Remove(tLogRootingMarker);
                    }
                }

                string firstTlog = _tlogFiles[0].ItemSpec;

                // empty all tlogs
                foreach (ITaskItem tlogFile in _tlogFiles)
                {
                    File.WriteAllText(tlogFile.ItemSpec, "", System.Text.Encoding.Unicode);
                }

                // Write out the dependency information as a new tlog
                using (StreamWriter newTlog = new StreamWriter(firstTlog, false, System.Text.Encoding.Unicode))
                {
                    foreach (string fileEntry in _dependencyTable.Keys)
                    {
                        // Give the task a chance to filter dependencies out of the written TLog
                        if (includeInTLog == null || includeInTLog(fileEntry))
                        {
                            // Write out the entry
                            newTlog.WriteLine(fileEntry);
                        }
                    }
                }
            }
            else if (_tlogMarker != String.Empty)
            {
                string markerDirectory = Path.GetDirectoryName(_tlogMarker);
                if (!Directory.Exists(markerDirectory))
                {
                    Directory.CreateDirectory(markerDirectory);
                }

                // There were no TLogs to save, so use the TLog marker
                // to create a marker file that can be used for up-to-date check.
                File.WriteAllText(_tlogMarker, "");
            }
        }

        /// <summary>
        /// Returns cached value for last write time of file. Update the cache if it is the first 
        /// time someone asking for that file
        /// </summary>
        public DateTime GetLastWriteTimeUtc(string file)
        {
            DateTime fileModifiedTimeUtc = DateTime.MinValue;
            if (!_lastWriteTimeUtcCache.TryGetValue(file, out fileModifiedTimeUtc))
            {
                fileModifiedTimeUtc = NativeMethods.GetLastWriteTimeUtc(file);
                _lastWriteTimeUtcCache[file] = fileModifiedTimeUtc;
            }

            return fileModifiedTimeUtc;
        }


        #endregion

        #region Static Methods

        /// <summary>
        /// Checks to see if the tracking data indicates that everything is up to date according to UpToDateCheckType.
        /// Note: If things are not up to date, then the TLogs are compacted to remove all entries in preparation to
        /// re-track execution of work.
        /// </summary>
        /// <param name="Log">TaskLoggingHelper from the host task</param>
        /// <param name="upToDateCheckType">UpToDateCheckType</param>
        /// <param name="readTLogNames">The array of read tlogs</param>
        /// <param name="writeTLogNames">The array of write tlogs</param>
        /// <returns></returns>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public static bool IsUpToDate(Task hostTask, UpToDateCheckType upToDateCheckType, ITaskItem[] readTLogNames, ITaskItem[] writeTLogNames)
        {
            bool isUpToDate;
            // Read the input graph (missing inputs are infinitely new - i.e. outputs are out of date)
            FlatTrackingData inputs = new FlatTrackingData(hostTask, readTLogNames, DateTime.MaxValue);

            // Read the output graph (missing outputs are infinitely old - i.e. outputs are out of date)
            FlatTrackingData outputs = new FlatTrackingData(hostTask, writeTLogNames, DateTime.MinValue);

            // Find out if we are up to date
            isUpToDate = FlatTrackingData.IsUpToDate(hostTask.Log, upToDateCheckType, inputs, outputs);

            // We're going to execute, so clear out the tlogs so
            // the new execution will correctly populate the tlogs a-new
            if (!isUpToDate)
            {
                // Remove all from inputs tlog
                inputs.DependencyTable.Clear();
                inputs.SaveTlog();

                // Remove all from outputs tlog
                outputs.DependencyTable.Clear();
                outputs.SaveTlog();
            }
            return isUpToDate;
        }

        /// <summary>
        /// Simple check of up to date state according to the tracking data and the UpToDateCheckType.
        /// Note: No tracking log compaction will take place when using this overload
        /// </summary>
        /// <param name="Log">TaskLoggingHelper from the host task</param>
        /// <param name="upToDateCheckType">UpToDateCheckType to use</param>
        /// <param name="inputs">FlatTrackingData structure containing the inputs</param>
        /// <param name="outputs">FlatTrackingData structure containing the outputs</param>
        /// <returns></returns>
        [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Log", Justification = "Has shipped as public API; plus it is a closer match to other locations in our codebase where 'Log' is a property and cased properly")]
        public static bool IsUpToDate(TaskLoggingHelper Log, UpToDateCheckType upToDateCheckType, FlatTrackingData inputs, FlatTrackingData outputs)
        {
            bool isUpToDate = false;
            // Keep a record of the task resources that was in use before
            ResourceManager taskResources = Log.TaskResources;

            Log.TaskResources = AssemblyResources.PrimaryResources;

            inputs.UpdateFileEntryDetails();
            outputs.UpdateFileEntryDetails();

            if (!inputs.TlogsAvailable || !outputs.TlogsAvailable || inputs.DependencyTable.Count == 0)
            {
                // 1) The TLogs are somehow missing, which means we need to build
                // 2) Because we are flat tracking, there are no roots which means that all the input file information 
                //    comes from the input Tlogs, if they are empty then we must build.
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_LogFilesNotAvailable");
            }
            else if (inputs.MissingFiles.Count > 0 || outputs.MissingFiles.Count > 0)
            {
                // Files are missing from either inputs or outputs, that means we need to build

                // Files are missing from inputs, that means we need to build
                if (inputs.MissingFiles.Count > 0)
                {
                    Log.LogMessageFromResources(MessageImportance.Low, "Tracking_MissingInputs");
                }
                // Too much logging leads to poor performance
                if (inputs.MissingFiles.Count > MaxLogCount)
                {
                    FileTracker.LogMessageFromResources(Log, MessageImportance.Low, "Tracking_InputsNotShown", inputs.MissingFiles.Count);
                }
                else
                {
                    // We have our set of inputs, log the details
                    foreach (string input in inputs.MissingFiles)
                    {
                        FileTracker.LogMessage(Log, MessageImportance.Low, "\t" + input);
                    }
                }

                // Files are missing from outputs, that means we need to build
                if (outputs.MissingFiles.Count > 0)
                {
                    Log.LogMessageFromResources(MessageImportance.Low, "Tracking_MissingOutputs");
                }
                // Too much logging leads to poor performance
                if (outputs.MissingFiles.Count > MaxLogCount)
                {
                    FileTracker.LogMessageFromResources(Log, MessageImportance.Low, "Tracking_OutputsNotShown", outputs.MissingFiles.Count);
                }
                else
                {
                    // We have our set of inputs, log the details
                    foreach (string output in outputs.MissingFiles)
                    {
                        FileTracker.LogMessage(Log, MessageImportance.Low, "\t" + output);
                    }
                }
            }
            else if (upToDateCheckType == UpToDateCheckType.InputOrOutputNewerThanTracking &&
                    (inputs.NewestFileTimeUtc > inputs.NewestTLogTimeUtc))
            {
                // One of the inputs is newer than the input tlog
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, inputs.NewestTLogFileName, inputs.NewestTLogTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputOrOutputNewerThanTracking &&
                    (outputs.NewestFileTimeUtc > outputs.NewestTLogTimeUtc))
            {
                // one of the outputs is newer than the output tlog
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", outputs.NewestFileName, outputs.NewestFileTimeUtc, outputs.NewestTLogFileName, outputs.NewestTLogTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputNewerThanOutput &&
                    (inputs.NewestFileTimeUtc > outputs.NewestFileTimeUtc))
            {
                // One of the inputs is newer than the outputs
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, outputs.NewestFileName, outputs.NewestFileTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputNewerThanTracking &&
                    (inputs.NewestFileTimeUtc > inputs.NewestTLogTimeUtc))
            {
                // One of the inputs is newer than the one of the TLogs
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, inputs.NewestTLogFileName, inputs.NewestTLogTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputNewerThanTracking &&
                    (inputs.NewestFileTimeUtc > outputs.NewestTLogTimeUtc))
            {
                // One of the inputs is newer than the one of the TLogs
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, outputs.NewestTLogFileName, outputs.NewestTLogTimeUtc);
            }
            else
            {
                // Nothing appears to have changed..
                isUpToDate = true;
                Log.LogMessageFromResources(MessageImportance.Normal, "Tracking_UpToDate");
            }

            // Set the task resources back now that we're done with it
            Log.TaskResources = taskResources;

            return isUpToDate;
        }

        /// <summary>
        /// Once tracked operations have been completed then we need to compact / finalize the Tlogs based
        /// on the success of the tracked execution. If it fails, then we clean out the TLogs. If it succeeds
        /// then we clean temporary files from the TLogs and re-write them.
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLogs", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public static void FinalizeTLogs(bool trackedOperationsSucceeded, ITaskItem[] readTLogNames, ITaskItem[] writeTLogNames, ITaskItem[] trackedFilesToRemoveFromTLogs)
        {
            // Read the input table, skipping missing files
            FlatTrackingData inputs = new FlatTrackingData(readTLogNames, true);

            // Read the output table, skipping missing files
            FlatTrackingData outputs = new FlatTrackingData(writeTLogNames, true);


            // If we failed we need to clean the Tlogs
            if (!trackedOperationsSucceeded)
            {
                // If the tool errors in some way, we assume that any and all inputs and outputs it wrote during
                // execution are wrong. So we compact the read and write tlogs to remove the entries for the
                // set of sources being compiled - the next incremental build will find no entries
                // and correctly cause the sources to be compiled
                // Remove all from inputs tlog
                inputs.DependencyTable.Clear();
                inputs.SaveTlog();

                // Remove all from outputs tlog
                outputs.DependencyTable.Clear();
                outputs.SaveTlog();
            }
            else
            {
                // If all went well with the tool execution, then compact the tlogs
                // to remove any files that are no longer on disk.
                // This removes any temporary files from the dependency graph

                // In addition to temporary file removal, an optional set of files to remove may be been supplied

                if (trackedFilesToRemoveFromTLogs != null && trackedFilesToRemoveFromTLogs.Length > 0)
                {
                    IDictionary<string, ITaskItem> trackedFilesToRemove = new Dictionary<string, ITaskItem>(StringComparer.OrdinalIgnoreCase);

                    foreach (ITaskItem removeFile in trackedFilesToRemoveFromTLogs)
                    {
                        trackedFilesToRemove.Add(FileUtilities.NormalizePath(removeFile.ItemSpec), removeFile);
                    }

                    // UNDONE: If necessary we could have two independent sets of "ignore" files, one for inputs and one for outputs
                    // Use an anonymous method to encapsulate the contains check for the output tlogs
                    outputs.SaveTlog(delegate (string fullTrackedPath)
                                     {
                                         // We need to answer the question "should fullTrackedPath be included in the TLog?"
                                         return (!trackedFilesToRemove.ContainsKey(fullTrackedPath));
                                     }
                    );

                    // Use an anonymous method to encapsulate the contains check for the input tlogs
                    inputs.SaveTlog(delegate (string fullTrackedPath)
                                     {
                                         // We need to answer the question "should fullTrackedPath be included in the TLog?"
                                         return (!trackedFilesToRemove.ContainsKey(fullTrackedPath));
                                     }
                    );
                }
                else
                {
                    // Compact the write tlog                        
                    outputs.SaveTlog();

                    // Compact the read tlog
                    inputs.SaveTlog();
                }
            }
        }
        #endregion
    }

    /// <summary>
    /// The possible types of up to date check that we can support
    /// </summary>
#if WHIDBEY_VISIBILITY
    internal
#else
    public
#endif
    enum UpToDateCheckType
    {
        InputNewerThanOutput,
        InputOrOutputNewerThanTracking,
        InputNewerThanTracking
    }
}
